import zipfile
from pathlib import Path

zip_path = Path("./data/GTsinger.zip")
extract_dir = Path("./data")

if not zip_path.exists():
    raise FileNotFoundError(f"Zip file not found: {zip_path}")

with zipfile.ZipFile(zip_path, 'r') as zip_ref:
    zip_ref.extractall(extract_dir)

print("✅ GTsinger.zip extracted successfully into data/")

✅ GTsinger.zip extracted successfully into data/

from collections import defaultdict
import random
import json
from pathlib import Path

# 1) Set the root directory
ROOT_DIR = Path("./data/GTsinger/English")
OUTPUT_JSON = ROOT_DIR / "metadata.json"
RANDOM_SEED = 42

# train/val/test dataset ratios
SPLIT_RATIOS = {
    "train": 0.7,
    "val": 0.15,
    "test": 0.15,
}

def technique_label(tech_folder: str, group_folder: str):
    # Basic techniques
    if tech_folder == "Breathy" and group_folder == "Breathy_Group":
        return "Breathy"
    if tech_folder == "Glissando" and group_folder == "Glissando_Group":
        return "Glissando"
    if tech_folder == "Pharyngeal" and group_folder == "Pharyngeal_Group":
        return "Pharyngeal"
    if tech_folder == "Vibrato" and group_folder == "Vibrato_Group":
        return "Vibrato"
    if tech_folder == "Mixed_Voice_and_Falsetto":
        if group_folder == "Falsetto_Group":
            return "Falsetto"
        if group_folder == "Mixed_Voice_Group":
            return "Mixed_Voice"
        return None

    return None

random.seed(RANDOM_SEED)

# samples_by_singer[label][singer_id] = [sample, sample, ...]
samples_by_singer = defaultdict(lambda: defaultdict(list))

# 2) Walk through all files
for wav in ROOT_DIR.rglob("*.wav"):
    # skip MacOS junk
    if "__MACOSX" in wav.parts:
        continue

    # make path relative to ROOT_DIR
    rel = wav.relative_to(ROOT_DIR)
    # [singer_id, tech_folder, song_title, group_folder, filename]
    parts = rel.parts

    if len(parts) != 5:
        print("Skipping unexpected path:", rel)
        continue

    singer_id, tech_folder, song_title, group_folder, filename = parts

    label = technique_label(tech_folder, group_folder)
    if label is None:
        continue

    sample = {
        "filepath": str(wav),
        "singer_id": singer_id,
        "tech_folder": tech_folder,
        "song_title": song_title,
        "group_folder": group_folder,
        "filename": filename,
        "technique": label,
    }

    samples_by_singer[label][singer_id].append(sample)

# 3) Split into Train, Valid, and Test
splits = {"train": [], "val": [], "test": []}

for label, singer_dict in samples_by_singer.items():
    for singer_id, samples in singer_dict.items():
        random.shuffle(samples)

        n = len(samples)
        n_train = int(n * SPLIT_RATIOS["train"])
        n_val = int(n * SPLIT_RATIOS["val"])
        n_test = n - n_train - n_val

        splits["train"].extend(samples[:n_train])
        splits["val"].extend(samples[n_train:n_train + n_val])
        splits["test"].extend(samples[n_train + n_val:])

# 4) Save a meta json file
with open(OUTPUT_JSON, "w", encoding="utf-8") as f:
    json.dump(splits, f, indent=4, ensure_ascii=False)

print(f"Saved metadata to {OUTPUT_JSON}")

Saved metadata to data/GTsinger/English/metadata.json

import matplotlib.pyplot as plt
import numpy as np

metadata_file = "./data/GTsinger/English/metadata.json"
splits = ["train", "val", "test"]

# Load metadata
meta_path = Path(metadata_file)
with open(meta_path, "r", encoding="utf-8") as f:
    all_meta = json.load(f)

print("=== Dataset Summary ===")

# Count samples per split
split_counts = {}
for split in splits:
    if split in all_meta:
        split_counts[split] = len(all_meta[split])
        print(f"{split:<10}: {len(all_meta[split])} samples")
    else:
        print(f"{split:<10}: NOT FOUND in metadata.json")

# singer_id -> [train, val, test]
singer_split_counts = defaultdict(lambda: {"train": 0, "val": 0, "test": 0})

for split in ["train", "val", "test"]:
    for item in all_meta[split]:
        singer_split_counts[item["singer_id"]][split] += 1

singers = sorted(singer_split_counts.keys())

train_counts = np.array([singer_split_counts[s]["train"] for s in singers])
val_counts   = np.array([singer_split_counts[s]["val"]   for s in singers])
test_counts  = np.array([singer_split_counts[s]["test"]  for s in singers])

# --- Plot (ABSOLUTE COUNTS) ---
plt.figure(figsize=(10, 5))

plt.bar(singers, train_counts, label="Train")
plt.bar(singers, val_counts,  bottom=train_counts, label="Val")
plt.bar(singers, test_counts, bottom=train_counts + val_counts, label="Test")

plt.ylabel("Number of Samples")
plt.title("Singer-wise Train / Val / Test Sample Counts")
plt.legend()
plt.grid(axis="y", linestyle="--", alpha=0.3)
plt.tight_layout()
plt.show()

# --- Plot split distribution ---
print(f"\n=== Number of Samples per Split ===")
plt.figure(figsize=(6, 4))
plt.bar(split_counts.keys(), split_counts.values(), color=["skyblue", "orange", "green"])
plt.title("Number of Samples per Split")
plt.ylabel("Samples")
plt.grid(axis="y", linestyle="--", alpha=0.3)
plt.show()

# --- Class distribution per split ---
for split in splits:
    if split not in all_meta:
        continue

    print(f"\n=== Class Distribution: {split.upper()} ===")
    items = all_meta[split]

    class_counts = defaultdict(int)
    for m in items:
        class_counts[m["technique"]] += 1

    # Print counts
    for tech, cnt in sorted(class_counts.items()):
        print(f"{tech:<15} : {cnt:4d}")

    # Bar chart
    plt.figure(figsize=(8, 4))
    plt.bar(class_counts.keys(), class_counts.values())
    plt.title(f"Class Distribution for {split}")
    plt.xlabel("Technique")
    plt.ylabel("Count")
    plt.xticks(rotation=45)
    plt.grid(axis="y", linestyle="--", alpha=0.3)
    plt.tight_layout()
    plt.show()

=== Dataset Summary ===
train     : 1925 samples
val       : 405 samples
test      : 432 samples

=== Number of Samples per Split ===

=== Class Distribution: TRAIN ===
Breathy         :  252
Falsetto        :  486
Glissando       :  246
Mixed_Voice     :  488
Pharyngeal      :  243
Vibrato         :  210

=== Class Distribution: VAL ===
Breathy         :   53
Falsetto        :  103
Glissando       :   51
Mixed_Voice     :  103
Pharyngeal      :   51
Vibrato         :   44

=== Class Distribution: TEST ===
Breathy         :   56
Falsetto        :  108
Glissando       :   56
Mixed_Voice     :  109
Pharyngeal      :   55
Vibrato         :   48

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from transformers import AutoProcessor, AutoModel
from typing import Optional, Literal
import librosa
import soundfile as sf
import numpy as np
import json
from pathlib import Path
from tqdm import tqdm
import os
import csv

class GTsingerAudioDataset(Dataset):
    """
    Dataset for GTsinger technique recognition

    mode = "mert"  -> returns MERT features:   [T, 1024]
    mode = "mel"   -> returns Mel spectrogram: [1, T, n_mels]
    """

    def __init__(
        self,
        metadata_file: str,
        split: str,
        mode: Literal["mert", "mel"] = "mert",

        # MERT-related (required if mode == "mert")
        mert_processor=None,
        mert_model=None,

        # Mel-spectrogram related (used if mode == "mel")
        n_mels: int = 128,

        # Audio settings
        max_length_sec: float = 10.0,
        target_sr: int = 24000,
        data_root: Optional[str] = None,

        seed: int = 42,
    ):
        # Reproducibility
        self.seed = seed
        random.seed(seed)
        np.random.seed(seed)

        assert mode in ["mert", "mel"], "mode must be 'mert' or 'mel'"
        self.mode = mode
        self.mert_processor = mert_processor
        self.mert_model = mert_model
        self.n_mels = n_mels
        self.max_length_sec = max_length_sec
        self.target_sr = target_sr

        # ---- metadata path ----
        metadata_path = Path(metadata_file)
        if not metadata_path.is_absolute():
            metadata_path = Path(metadata_file).absolute()

        with open(metadata_path, "r", encoding="utf-8") as f:
            all_meta = json.load(f)

        if split not in all_meta:
            raise ValueError(
                f"Split '{split}' not found in {metadata_path}. "
                f"Available splits: {list(all_meta.keys())}"
            )

        all_samples = []
        for k, v in all_meta.items():
            if isinstance(v, list):
                all_samples.extend(v)

        techniques = sorted({m["technique"] for m in all_samples})
        self.technique_map = {tech: idx for idx, tech in enumerate(techniques)}
        self.idx_to_tech = {idx: tech for tech, idx in self.technique_map.items()}
        self.num_classes = len(self.technique_map)

        # print("[Dataset] Technique → index mapping:")
        # for tech, idx in self.technique_map.items():
        #     print(f"  {tech:<15} -> {idx}")

        # ---- split based on medadata ----
        self.metadata = all_meta[split]
        self.split = split

        # ---- data_root ----
        if data_root is not None:
            self.data_root = Path(data_root)
        else:
            self.data_root = Path(".").absolute()

        print(f"[Dataset] Split '{split}': {len(self.metadata)} samples")

        if self.mode == "mert":
            assert self.mert_processor is not None and self.mert_model is not None, \
                "mert_processor and mert_model are required when mode='mert'"

    def _resolve_audio_path(self, rel_or_abs: str) -> Path:
        p = Path(rel_or_abs)
        if p.is_absolute():
            return p
        return self.data_root / p

    def _load_and_fix_length(self, audio_path: Path) -> np.ndarray:
        try:
            audio, sr = librosa.load(str(audio_path), sr=self.target_sr, mono=True)

            # amplitude normalize
            if audio.max() > 0:
                audio = audio / np.abs(audio).max()

            target_len = int(self.max_length_sec * self.target_sr)
            if audio.shape[0] > target_len:
                audio = audio[:target_len]
            elif audio.shape[0] < target_len:
                audio = np.pad(audio, (0, target_len - audio.shape[0]), mode="constant")

        except Exception as e:
            print(f"[Dataset:{self.split}] Error loading {audio_path}: {e}")
            audio = np.random.randn(int(self.max_length_sec * self.target_sr)).astype(np.float32)

        return audio.astype(np.float32)

    def __len__(self):
        return len(self.metadata)

    def __getitem__(self, idx):
        meta = self.metadata[idx]
        filepath = meta["filepath"]
        audio_path = self._resolve_audio_path(filepath)
        audio = self._load_and_fix_length(audio_path)

        # Labels
        tech_name = meta["technique"]
        if tech_name not in self.technique_map:
            raise ValueError(f"Unknown technique '{tech_name}' in meta.json")
        label = torch.tensor(self.technique_map[tech_name], dtype=torch.long)

        # 1) Mel spectrogram (without a pretrained model)
        if self.mode == "mel":
            mel_spec = librosa.feature.melspectrogram(
                y=audio,
                sr=self.target_sr,
                n_fft=2048,
                hop_length=512,
                n_mels=self.n_mels,
                fmin=50,
                fmax=8000,
            )
            mel_spec = librosa.power_to_db(mel_spec, ref=np.max)

            # per-sample normalize
            mel_spec = (mel_spec - mel_spec.mean()) / (mel_spec.std() + 1e-9)

            # mel_spec: numpy [n_mels, time]
            mel_spec = torch.from_numpy(mel_spec).float()   # [n_mels, T]
            mel_spec = mel_spec.transpose(0, 1)             # [T, n_mels]

            return {
                "features": mel_spec,   # [T, n_mels]
                "label": label,
                "filepath": filepath,
            }

        # 2) MERT processor (with a pretrained model)
        elif self.mode == "mert":
            with torch.no_grad():
                inputs = self.mert_processor(
                    audio,
                    sampling_rate=self.target_sr,
                    return_tensors="pt",
                )
                # device
                inputs = {k: v.to(self.mert_model.device) for k, v in inputs.items()}
                hidden_states = self.mert_model(**inputs).last_hidden_state  # [1, T, 1024]

            return {
                "features": hidden_states.squeeze(0),  # [T, 1024]
                "label": label,
                "filepath": filepath,
            }

/home/steve/anaconda3/envs/DL/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

class CNN1D_Classifier(nn.Module):
    """
    1D CNN classifier over time for frame-level features.
    """
    def __init__(self, in_channels, hidden=128, kernel_size=5, n_classes=6, dropout=0.3):
        super().__init__()

        self.conv_layers = nn.Sequential(
            nn.Conv1d(in_channels, hidden, kernel_size=kernel_size,
                      padding=kernel_size // 2),
            nn.BatchNorm1d(hidden),
            nn.GELU(),
            nn.Dropout(dropout),
        )

        self.classifier = nn.Sequential(
            nn.Dropout(dropout),
            nn.Linear(hidden, hidden // 2),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden // 2, n_classes),
        )

    def forward(self, z_btf: torch.Tensor):
        # z_btf: [B, T, F]
        x = z_btf.transpose(1, 2)      # [B, F, T]
        x = self.conv_layers(x)        # [B, hidden, T]
        x = x.mean(dim=2)              # [B, hidden]
        return self.classifier(x)      # [B, n_classes]

class CRNN_Classifier(nn.Module):
    """
    CRNN classifier over time for frame-level features.
    """
    def __init__(self, in_channels, cnn_hidden=128, rnn_hidden=128, rnn_layers=2, n_classes=6, dropout=0.3):
        super().__init__()

        self.conv = nn.Sequential(
            nn.Conv1d(in_channels, cnn_hidden, kernel_size=5, padding=2),
            nn.BatchNorm1d(cnn_hidden),
            nn.GELU(),
            nn.Dropout(dropout),
        )

        self.rnn = nn.LSTM(
            input_size=cnn_hidden,
            hidden_size=rnn_hidden,
            num_layers=rnn_layers,
            batch_first=True,
            bidirectional=True,
            dropout=dropout if rnn_layers > 1 else 0.0,
        )

        self.classifier = nn.Sequential(
            nn.Dropout(dropout),
            nn.Linear(rnn_hidden * 2, n_classes),
        )

    def forward(self, z_btf: torch.Tensor):
        # z_btf: [B, T, F]
        x = z_btf.transpose(1, 2)      # [B, F, T]
        x = self.conv(x)               # [B, cnn_hidden, T]
        x = x.transpose(1, 2)          # [B, T, cnn_hidden]
        x, _ = self.rnn(x)             # [B, T, 2*rnn_hidden]
        x = x.mean(dim=1)              # [B, 2*rnn_hidden]
        return self.classifier(x)      # [B, n_classes]

class Transformer_Classifier(nn.Module):
    """
    Transformer classifier over time for frame-level features.
    """
    def __init__(self, in_channels, d_model=128, nhead=4, n_classes=6, dropout=0.3):
        super().__init__()

        self.input_proj = nn.Linear(in_channels, d_model)  # F -> d_model

        encoder_layer = nn.TransformerEncoderLayer(
            d_model=d_model,
            nhead=nhead,
            dim_feedforward=d_model * 2,
            dropout=dropout,
            batch_first=True,
        )

        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=1)

        self.classifier = nn.Sequential(
            nn.Dropout(dropout),
            nn.Linear(d_model, n_classes),
        )

    def forward(self, z_btc: torch.Tensor):
        # z_btc: [B, T, F]
        x = self.input_proj(z_btc)      # [B, T, d_model]
        x = self.transformer(x)         # [B, T, d_model]
        x = x.mean(dim=1)               # global average over time -> [B, d_model]
        return self.classifier(x)       # [B, n_classes]

import matplotlib.pyplot as plt

def plot_history(history, results_dir: Path):
    results_dir = Path(results_dir)
    results_dir.mkdir(parents=True, exist_ok=True)

    epochs = history["epoch"]

    # Loss curve
    plt.figure(figsize=(6,4))
    plt.plot(epochs, history["train_loss"], label="Train Loss")
    plt.plot(epochs, history["val_loss"], label="Val Loss")
    plt.xlabel("Epoch")
    plt.ylabel("Loss")
    plt.legend()
    plt.grid(alpha=0.3)
    plt.tight_layout()
    plt.savefig(results_dir / "loss_curve.png", dpi=150)
    plt.close()

    # Accuracy curve
    plt.figure(figsize=(6,4))
    plt.plot(epochs, history["train_acc"], label="Train Acc")
    plt.plot(epochs, history["val_acc"], label="Val Acc")
    plt.xlabel("Epoch")
    plt.ylabel("Accuracy")
    plt.legend()
    plt.grid(alpha=0.3)
    plt.tight_layout()
    plt.savefig(results_dir / "accuracy_curve.png", dpi=150)
    plt.close()

def save_history_csv(history, results_dir: Path):
    results_dir = Path(results_dir)
    csv_path = results_dir / "metrics_history.csv"

    with open(csv_path, "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["epoch", "train_loss", "train_acc", "val_loss", "val_acc"])
        for i in range(len(history["epoch"])):
            writer.writerow([
                history["epoch"][i],
                history["train_loss"][i],
                history["train_acc"][i],
                history["val_loss"][i],
                history["val_acc"][i],
            ])

def run_epoch(model, dataloader, criterion, optimizer, device, train: bool):
    if train:
        model.train()
    else:
        model.eval()

    total_loss = 0.0
    correct = 0
    total = 0

    for batch in tqdm(dataloader, desc="Train" if train else "Val", leave=False):
        features = batch["features"].to(device)   # [B, T, F]
        labels = batch["label"].to(device)        # [B]

        if train:
            optimizer.zero_grad()

        with torch.set_grad_enabled(train):
            logits = model(features)              # [B, n_classes]
            loss = criterion(logits, labels)

            if train:
                loss.backward()
                optimizer.step()

        total_loss += loss.item()
        preds = logits.argmax(dim=1)
        correct += (preds == labels).sum().item()
        total += labels.size(0)

    avg_loss = total_loss / max(1, len(dataloader))
    acc = correct / max(1, total)
    return avg_loss, acc

def train_gtsinger(
    metadata_file: str,
    data_root: str,
    mode: str = "mel",        # "mel: no pre-trained model" or "mert: pre-trained model"
    model_type: str = "cnn",  # Classifiers ["cnn", "crnn", "transformer"]
    n_mels: int = 128,
    batch_size: int = 32,
    epochs: int = 100,
    lr: float = 1e-3,
    weight_decay: float = 0.0,
    dropout: float = 0.3,
    results_dir: str = "./results",
    seed: int = 42,
):

    # Reproducibility
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

    assert mode in ["mel", "mert"]
    assert model_type in ["cnn", "crnn", "transformer"]

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(f"\n=== Training GTsinger ({mode}, {model_type}) on {device} ===")

    # Pretrained model selection
    mert_processor, mert_model = None, None
    if mode == "mert":
        print("\nLoading MERT model...")
        model_name = "m-a-p/MERT-v1-330M"
        mert_processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
        mert_model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
        mert_model.to(device)
        mert_model.eval()
    else:
        print("\nMode = mel → MERT not loaded.")

    # Dataset / DataLoader
    metadata_file = str(metadata_file)
    data_root = str(data_root)

    print("\nCreating datasets...")
    train_dataset = GTsingerAudioDataset(
        metadata_file=metadata_file,
        split="train",
        mode=mode,
        mert_processor=mert_processor,
        mert_model=mert_model,
        n_mels=n_mels,
        max_length_sec=10.0,
        target_sr=24000,
        data_root=data_root,
    )

    val_dataset = GTsingerAudioDataset(
        metadata_file=metadata_file,
        split="val",
        mode=mode,
        mert_processor=mert_processor,
        mert_model=mert_model,
        n_mels=n_mels,
        max_length_sec=10.0,
        target_sr=24000,
        data_root=data_root,
    )

    # features shape: [T, F] → F
    sample_feat = train_dataset[0]["features"]   # [T, F]
    in_channels = sample_feat.shape[-1]
    n_classes = train_dataset.num_classes
    print(f"\nInput feature dim (F): {in_channels}")
    print(f"Num classes: {n_classes}")

    # Data loading reproducibility
    g = torch.Generator()
    g.manual_seed(seed)

    # DataLoaders - Batch
    train_loader = DataLoader(
        train_dataset,
        batch_size=batch_size,
        shuffle=True,
        worker_init_fn=lambda worker_id: np.random.seed(seed + worker_id),
        generator=g
    )

    val_loader = DataLoader(
        val_dataset,
        batch_size=batch_size,
        shuffle=False,
        worker_init_fn=lambda worker_id: np.random.seed(seed + worker_id),
        generator=g
    )

    # Classifier Models
    print(f"\nInitializing {model_type.upper()} model...")

    if model_type == "cnn":
        model = CNN1D_Classifier(
            in_channels=in_channels,
            hidden=128,
            kernel_size=5,
            n_classes=n_classes,
            dropout=dropout,
        ).to(device)

    elif model_type == "crnn":
        model = CRNN_Classifier(
            in_channels=in_channels,
            cnn_hidden=128,
            rnn_hidden=128,
            rnn_layers=2,
            n_classes=n_classes,
            dropout=dropout,
        ).to(device)

    elif model_type == "transformer":
        model = Transformer_Classifier(
            in_channels=in_channels,
            d_model=128,
            nhead=4,
            n_classes=n_classes,
            dropout=dropout,
        ).to(device)

    total_params = sum(p.numel() for p in model.parameters())
    print(f"Model parameters: {total_params:,}")

    # Loss / Optim / Scheduler
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
    scheduler = optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, mode="min", patience=3, factor=0.5)

    # Results dir
    results_dir = Path(results_dir) / f"{mode}_{model_type}"
    results_dir.mkdir(parents=True, exist_ok=True)


    # Label mapping
    with open(results_dir / "label_mapping.json", "w", encoding="utf-8") as f:
        json.dump(train_dataset.idx_to_tech, f, indent=2)

    # Saving training history
    history = {
        "epoch": [],
        "train_loss": [],
        "train_acc": [],
        "val_loss": [],
        "val_acc": [],
    }
    best_val_loss = float("inf")
    best_model_path = results_dir / "best_model.pth"

    print(f"\nStart training for {epochs} epochs...")
    for epoch in range(1, epochs + 1):
        print(f"\nEpoch {epoch}/{epochs}")

        train_loss, train_acc = run_epoch(
            model, train_loader, criterion, optimizer, device, train=True
        )
        val_loss, val_acc = run_epoch(
            model, val_loader, criterion, optimizer, device, train=False
        )

        print(f"Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}")
        print(f"Val   Loss: {val_loss:.4f}, Val   Acc: {val_acc:.4f}")

        history["epoch"].append(epoch)
        history["train_loss"].append(train_loss)
        history["train_acc"].append(train_acc)
        history["val_loss"].append(val_loss)
        history["val_acc"].append(val_acc)

        scheduler.step(val_loss)

        # checkpoint every 10 epochs
        if epoch % 10 == 0:
          torch.save(model.state_dict(), results_dir / f"epoch_{epoch:03d}.pth")

        # best model
        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(model.state_dict(), best_model_path)
            print(f"✓ New best model saved (val_loss={best_val_loss:.4f})")

        # save csv/plot and update at every epoch
        save_history_csv(history, results_dir)
        plot_history(history, results_dir)

    print("\n=== Training finished ===")
    print(f"Best val loss: {best_val_loss:.4f}")
    print(f"Best model path: {best_model_path.resolve()}")

    epochs = history["epoch"]

    # Loss curve
    plt.figure(figsize=(6, 4))
    plt.plot(epochs, history["train_loss"], label="Train Loss")
    plt.plot(epochs, history["val_loss"], label="Val Loss")
    plt.xlabel("Epoch")
    plt.ylabel("Loss")
    plt.legend()
    plt.grid(alpha=0.3)
    plt.tight_layout()
    plt.show()

    # Accuracy curve
    plt.figure(figsize=(6, 4))
    plt.plot(epochs, history["train_acc"], label="Train Acc")
    plt.plot(epochs, history["val_acc"], label="Val Acc")
    plt.xlabel("Epoch")
    plt.ylabel("Accuracy")
    plt.legend()
    plt.grid(alpha=0.3)
    plt.tight_layout()
    plt.show()

    return model, history

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (1) No-pretrained model + 1DCNN Classifier
mode = "mel"
model_type = "cnn"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=100,
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mel, cnn) on cuda ===

Mode = mel → MERT not loaded.

Creating datasets...
[Dataset] Split 'train': 1925 samples
[Dataset] Split 'val': 405 samples

Input feature dim (F): 128
Num classes: 6

Initializing CNN model...
Model parameters: 90,950

Results dir: /home/steve/final_report/train/GT_results/mel_cnn

Start training for 100 epochs...

Epoch 1/100

Train Loss: 1.4407, Train Acc: 0.4577
Val   Loss: 1.1635, Val   Acc: 0.5704
✓ New best model saved (val_loss=1.1635)

Epoch 2/100

Train Loss: 1.2070, Train Acc: 0.5273
Val   Loss: 1.0910, Val   Acc: 0.6025
✓ New best model saved (val_loss=1.0910)

Epoch 3/100

Train Loss: 1.1004, Train Acc: 0.5673
Val   Loss: 1.0108, Val   Acc: 0.6173
✓ New best model saved (val_loss=1.0108)

Epoch 4/100

Train Loss: 1.0369, Train Acc: 0.5891
Val   Loss: 0.9515, Val   Acc: 0.6346
✓ New best model saved (val_loss=0.9515)

Epoch 5/100

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (2) No-pretrained model + CRNN Classifier
mode = "mel"
model_type = "crnn"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=100,
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mel, crnn) on cuda ===

Mode = mel → MERT not loaded.

Creating datasets...
[Dataset] Split 'train': 1925 samples
[Dataset] Split 'val': 405 samples

Input feature dim (F): 128
Num classes: 6

Initializing CRNN model...
Model parameters: 743,302

Start training for 100 epochs...

Epoch 1/100

Train Loss: 1.3860, Train Acc: 0.4571
Val   Loss: 1.1799, Val   Acc: 0.5407
✓ New best model saved (val_loss=1.1799)

Epoch 2/100

Train Loss: 1.1165, Train Acc: 0.5652
Val   Loss: 1.5074, Val   Acc: 0.4716

Epoch 3/100

Train Loss: 1.0158, Train Acc: 0.5938
Val   Loss: 0.9350, Val   Acc: 0.6296
✓ New best model saved (val_loss=0.9350)

Epoch 4/100

Train Loss: 0.9330, Train Acc: 0.6197
Val   Loss: 0.9489, Val   Acc: 0.6296

Epoch 5/100

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (3) No-pretrained model + Transformer Classifier
mode = "mel"
model_type = "transformer"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=100,
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mel, transformer) on cuda ===

Mode = mel → MERT not loaded.

Creating datasets...
[Dataset] Split 'train': 1930 samples
[Dataset] Split 'val': 412 samples

Input feature dim (F): 128
Num classes: 6

Initializing TRANSFORMER model...
Model parameters: 149,766

Results dir: /home/steve/final_report/train/GT_results/mel_transformer

Start training for 100 epochs...

Epoch 1/100

Train Loss: 1.3751, Train Acc: 0.4560
Val   Loss: 1.1690, Val   Acc: 0.5510
✓ New best model saved (val_loss=1.1690)

Epoch 2/100

Train Loss: 1.1205, Train Acc: 0.5627
Val   Loss: 1.0550, Val   Acc: 0.5995
✓ New best model saved (val_loss=1.0550)

Epoch 3/100

Train Loss: 1.0027, Train Acc: 0.6036
Val   Loss: 0.9678, Val   Acc: 0.6092
✓ New best model saved (val_loss=0.9678)

Epoch 4/100

Train Loss: 0.9529, Train Acc: 0.6197
Val   Loss: 0.9534, Val   Acc: 0.6481
✓ New best model saved (val_loss=0.9534)

Epoch 5/100

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (4) Pretrained model (MERT) + 1DCNN Classifier
mode = "mert"
model_type = "cnn"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=50, # 50 epochs due to the fast convergence
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mert, cnn) on cuda ===

Loading MERT model...

Creating datasets...
[Dataset] Split 'train': 1925 samples
[Dataset] Split 'val': 405 samples

Input feature dim (F): 1024
Num classes: 6

Initializing CNN model...
Model parameters: 664,390

Start training for 50 epochs...

Epoch 1/50

Train Loss: 1.4051, Train Acc: 0.4603
Val   Loss: 1.1098, Val   Acc: 0.5728
✓ New best model saved (val_loss=1.1098)

Epoch 2/50

Train Loss: 1.0134, Train Acc: 0.6223
Val   Loss: 0.8714, Val   Acc: 0.6568
✓ New best model saved (val_loss=0.8714)

Epoch 3/50

Train Loss: 0.8289, Train Acc: 0.6784
Val   Loss: 0.8233, Val   Acc: 0.6840
✓ New best model saved (val_loss=0.8233)

Epoch 4/50

Train Loss: 0.7322, Train Acc: 0.7195
Val   Loss: 0.6611, Val   Acc: 0.7506
✓ New best model saved (val_loss=0.6611)

Epoch 5/50

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (5) Pretrained model (MERT) + CRNN Classifier
mode = "mert"
model_type = "crnn"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=50, # 50 epochs due to the fast convergence
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mert, crnn) on cuda ===

Loading MERT model...

Creating datasets...
[Dataset] Split 'train': 1925 samples
[Dataset] Split 'val': 405 samples

Input feature dim (F): 1024
Num classes: 6

Initializing CRNN model...
Model parameters: 1,316,742

Start training for 50 epochs...

Epoch 1/50

Train Loss: 1.2794, Train Acc: 0.4930
Val   Loss: 1.0148, Val   Acc: 0.5926
✓ New best model saved (val_loss=1.0148)

Epoch 2/50

Train Loss: 0.9073, Train Acc: 0.6499
Val   Loss: 0.8656, Val   Acc: 0.6593
✓ New best model saved (val_loss=0.8656)

Epoch 3/50

Train Loss: 0.7245, Train Acc: 0.7158
Val   Loss: 0.7562, Val   Acc: 0.6988
✓ New best model saved (val_loss=0.7562)

Epoch 4/50

Train Loss: 0.6884, Train Acc: 0.7356
Val   Loss: 0.6256, Val   Acc: 0.7654
✓ New best model saved (val_loss=0.6256)

Epoch 5/50

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"   # root dir

# (6) Pretrained model (MERT) + Transformer Classifier
mode = "mert"
model_type = "transformer"

model, history = train_gtsinger(
    metadata_file=metadata_file,
    data_root=data_root,
    mode=mode,
    model_type=model_type,
    n_mels=128,
    batch_size=32,
    epochs=50, # 50 epochs due to the fast convergence
    lr=1e-3,
    dropout=0.3,
    seed=42,
    results_dir=f"./train/GT_results",
)

=== Training GTsinger (mert, transformer) on cuda ===

Loading MERT model...

Creating datasets...
[Dataset] Split 'train': 1925 samples
[Dataset] Split 'val': 405 samples

Input feature dim (F): 1024
Num classes: 6

Initializing TRANSFORMER model...
Model parameters: 264,454

Start training for 50 epochs...

Epoch 1/50

Train Loss: 1.4753, Train Acc: 0.4229
Val   Loss: 1.1259, Val   Acc: 0.5259
✓ New best model saved (val_loss=1.1259)

Epoch 2/50

Train Loss: 1.0293, Train Acc: 0.6078
Val   Loss: 1.1282, Val   Acc: 0.5333

Epoch 3/50

Train Loss: 0.8779, Train Acc: 0.6613
Val   Loss: 0.8004, Val   Acc: 0.6815
✓ New best model saved (val_loss=0.8004)

Epoch 4/50

Train Loss: 0.7605, Train Acc: 0.6764
Val   Loss: 0.7859, Val   Acc: 0.7012
✓ New best model saved (val_loss=0.7859)

Epoch 5/50

def load_trained_model(model_path: str, mode: str = "mert", model_type: str = "cnn", device: str = "cuda"):
    """
    Load a trained model from a checkpoint.

    Args:
        model_path: Path to the saved model checkpoint (.pth file)
        mode: Feature extraction mode ("mert" or "mel")
        model_type: Classifier type ("cnn", "crnn", or "transformer")
        device: Device to load the model on

    Returns:
        Loaded model ready for inference
    """
    # Load label mapping
    results_dir = Path(model_path).parent
    with open(results_dir / "label_mapping.json", "r") as f:
        label_mapping = json.load(f)
    id_to_label = {int(k): v for k, v in label_mapping.items()}

    # Initialize model based on mode
    if mode == "mert":
        from transformers import AutoProcessor, AutoModel
        model_name = "m-a-p/MERT-v1-330M"
        mert_processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
        mert_model = AutoModel.from_pretrained(model_name, trust_remote_code=True).to(device)
        mert_model.eval()
        feature_dim = 1024
    else:
        mert_processor = None
        mert_model = None
        feature_dim = 128  # For mel spectrogram

    # Initialize classifier
    if model_type == "cnn":
        classifier = CNN1D_Classifier(in_channels=feature_dim, n_classes=6, dropout=0.3)
    elif model_type == "crnn":
        classifier = CRNN_Classifier(in_channels=feature_dim, n_classes=6, dropout=0.3)
    elif model_type == "transformer":
        classifier = Transformer_Classifier(in_channels=feature_dim, n_classes=6, dropout=0.3)
    else:
        raise ValueError(f"Unknown model_type: {model_type}")

    # Load trained weights
    classifier.load_state_dict(torch.load(model_path, map_location=device))
    classifier = classifier.to(device)
    classifier.eval()

    return classifier, mert_model, mert_processor, id_to_label

from sklearn.metrics import confusion_matrix, precision_recall_fscore_support

def model_test(classifier, test_loader, device, id_to_label=None, results_dir: str | Path = "./results_test"):
    """
    Evaluate model on test_loader:
      - Compute accuracy
      - Compute normalized confusion matrix (per-class)
      - Save confusion matrix as PNG and also display it
      - Compute per-class Precision / Recall / F1 and print as a table
    """
    classifier.eval()
    total, correct = 0, 0
    all_preds, all_labels = [], []

    with torch.no_grad():
        for batch in tqdm(test_loader, desc="Testing"):
            features = batch["features"].to(device)  # [B, T, F]
            labels = batch["label"].to(device)

            logits = classifier(features)           # [B, n_classes]
            preds = logits.argmax(dim=1)

            correct += (preds == labels).sum().item()
            total += labels.size(0)

            all_preds.extend(preds.cpu().tolist())
            all_labels.extend(labels.cpu().tolist())

    acc = correct / total
    print(f"\nTest accuracy: {acc:.4f}")

    # ----- Confusion Matrix -----
    results_dir = Path(results_dir)
    results_dir.mkdir(parents=True, exist_ok=True)

    # decide class ids & names
    if id_to_label is not None:
        class_ids = sorted(id_to_label.keys())
        class_names = [id_to_label[i] for i in class_ids]
    else:
        class_ids = sorted(set(all_labels) | set(all_preds))
        class_names = [str(i) for i in class_ids]

    # confusion_matrix per true class
    cm = confusion_matrix(
        all_labels,
        all_preds,
        labels=class_ids,
        normalize="true",
    )

    # Plot confusion matrix
    plt.figure(figsize=(6, 5))
    im = plt.imshow(cm, interpolation="nearest", cmap="Blues")
    plt.title("Normalized Confusion Matrix")
    plt.colorbar(im, fraction=0.046, pad=0.04)
    tick_marks = np.arange(len(class_names))
    plt.xticks(tick_marks, class_names, rotation=45, ha="right")
    plt.yticks(tick_marks, class_names)
    plt.ylabel("True Label")
    plt.xlabel("Predicted Label")

    # annotate each cell
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            value = cm[i, j]
            plt.text(
                j, i,
                f"{value:.2f}",
                ha="center",
                va="center",
                color="white" if value > 0.5 else "black",
                fontsize=8,
            )

    plt.tight_layout()
    cm_path = results_dir / "confusion_matrix_normalized.png"
    plt.savefig(cm_path, dpi=150)
    plt.show()
    print(f"Saved normalized confusion matrix to: {cm_path}")

    # ----- Precision / Recall / F1 per class -----
    precision, recall, f1, support = precision_recall_fscore_support(
        all_labels,
        all_preds,
        labels=class_ids,
        zero_division=0,
    )

    # ===== Per-class Accuracy =====
    class_accuracy = []
    for cid in class_ids:
        cls_mask = np.array(all_labels) == cid
        cls_correct = (np.array(all_preds)[cls_mask] == cid).sum()
        cls_total = cls_mask.sum()
        acc = cls_correct / max(1, cls_total)
        class_accuracy.append(acc)

    # ===== Print Full Metrics Table =====
    print("\n Per-class Evaluation Metrics:")
    header = f"{'Class':<15} {'Support':>7} {'Accuracy':>10} {'Precision':>10} {'Recall':>10} {'F1-score':>10}"
    print(header)
    print("-" * len(header))

    for cid, p, r, f, s, a in zip(class_ids, precision, recall, f1, support, class_accuracy):
        name = id_to_label[cid] if id_to_label else str(cid)
        print(f"{name:<15} {s:7d} {a:10.3f} {p:10.3f} {r:10.3f} {f:10.3f}")

    # macro-average
    macro_p = precision.mean()
    macro_r = recall.mean()
    macro_f = f1.mean()
    print("-" * len(header))
    print(f"{'Macro-avg':<15} {sum(support):7d} {macro_p:10.3f} {macro_r:10.3f} {macro_f:10.3f}")

    return acc, all_preds, all_labels

metadata_file = "./data/GTsinger/English/metadata.json"
data_root = "./"

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

mode = "mert"
model_type = "cnn"     # "cnn" / "crnn" / "transformer"
model_path = f"./train/GT_results/{mode}_{model_type}/best_model.pth"

print("\nLoading MERT model...")
model_name = "m-a-p/MERT-v1-330M"
mert_processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
mert_model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
mert_model.to(device)
mert_model.eval()

test_dataset = GTsingerAudioDataset(
    metadata_file=metadata_file,
    split="test",
    mode=mode,
    mert_processor=mert_processor,
    mert_model=mert_model,
    n_mels=128,
    max_length_sec=10.0,
    target_sr=24000,
    data_root=data_root,
)

test_loader = DataLoader(
    test_dataset,
    batch_size=32,
    shuffle=False,
)

# Load the trained model
classifier, mert_model, mert_processor, id_to_label = load_trained_model(
    model_path=model_path,
    mode=mode,
    model_type=model_type,
    device=device,
)

# Test the model
test_acc, test_preds, test_labels = model_test(
    classifier,
    test_loader,
    device,
    id_to_label=id_to_label,
    results_dir=f"./test/{mode}_{model_type}_test_results",
)

Loading MERT model...
[Dataset] Split 'test': 432 samples

Testing: 100%|██████████| 14/14 [00:08<00:00,  1.60it/s]

Test accuracy: 0.8843

Saved normalized confusion matrix to: test/mert_cnn_test_results/confusion_matrix_normalized.png

 Per-class Evaluation Metrics:
Class           Support   Accuracy  Precision     Recall   F1-score
-------------------------------------------------------------------
Breathy              56      0.839      0.839      0.839      0.839
Falsetto            108      0.889      0.950      0.889      0.919
Glissando            56      0.804      0.776      0.804      0.789
Mixed_Voice         109      0.908      0.868      0.908      0.888
Pharyngeal           55      0.909      0.909      0.909      0.909
Vibrato              48      0.938      0.938      0.938      0.938
-------------------------------------------------------------------
Macro-avg           432      0.880      0.881      0.880

from IPython.display import Audio, display

demo_root = Path("./data/demo_data")

# All wav files in demo_root
wav_files = sorted(demo_root.rglob("*.wav"))

print(f"Found {len(wav_files)} demo wav files:\n")

for wav_path in wav_files:
    label = wav_path.parent.name
    print(f"▶ {label}: {wav_path.name}")
    display(Audio(filename=str(wav_path), autoplay=False))

Found 4 demo wav files:

▶ Breathy: breathy_1.wav

▶ Breathy: breathy_2.wav

▶ Vibrato: vibrato_1.wav

▶ Vibrato: vibrato_2.wav

# ======= Demo data loader with the same preprocessing =======
class DemoDataset(Dataset):
    def __init__(self, demo_root: str, mert_processor, mert_model, id_to_label,
                 max_length_sec: float = 10.0, target_sr: int = 24000):

        self.demo_root = Path(demo_root)
        self.mert_processor = mert_processor
        self.mert_model = mert_model
        self.max_length_sec = max_length_sec
        self.target_sr = target_sr

        self.label_to_id = {v: int(k) for k, v in id_to_label.items()}

        self.samples = []
        for label_name in sorted(os.listdir(self.demo_root)):
            class_dir = self.demo_root / label_name
            if not class_dir.is_dir():
                continue

            for wav in class_dir.glob("*.wav"):
                self.samples.append({
                    "filepath": wav,
                    "label_name": label_name,
                })

        print(f"Demo File Loaded {len(self.samples)} demo samples")

    def __len__(self):
        return len(self.samples)

    def _load_and_fix_length(self, audio_path: Path):
        audio, sr = librosa.load(str(audio_path), sr=self.target_sr, mono=True)

        if audio.max() > 0:
            audio = audio / np.abs(audio).max()

        target_len = int(self.max_length_sec * self.target_sr)
        if audio.shape[0] > target_len:
            audio = audio[:target_len]
        elif audio.shape[0] < target_len:
            audio = np.pad(audio, (0, target_len - audio.shape[0]), mode="constant")

        return audio.astype(np.float32)

    def __getitem__(self, idx):
        meta = self.samples[idx]
        audio = self._load_and_fix_length(meta["filepath"])

        label_name = meta["label_name"]
        label_id = self.label_to_id[label_name]

        with torch.no_grad():
            inputs = self.mert_processor(
                audio,
                sampling_rate=self.target_sr,
                return_tensors="pt",
            )
            inputs = {k: v.to(self.mert_model.device) for k, v in inputs.items()}
            hidden_states = self.mert_model(**inputs).last_hidden_state  # [1, T, 1024]

        features = hidden_states.squeeze(0)  # [T, 1024]

        return {
            "features": features,
            "label": torch.tensor(label_id, dtype=torch.long),
            "filepath": str(meta["filepath"]),
            "label_name": label_name,
        }


def model_demo_test(classifier, test_loader, device, id_to_label):
    classifier.eval()

    with torch.no_grad():
        for batch in test_loader:
            features = batch["features"].to(device)
            labels = batch["label"].to(device)

            logits = classifier(features)
            preds = logits.argmax(dim=1)

            for p, gt in zip(preds.cpu(), labels.cpu()):
                print(
                    f"GT: {id_to_label[gt.item()]} | "
                    f"Pred: {id_to_label[p.item()]}"
                )

# Loading the demo data
demo_root = "./data/demo_data"

demo_dataset = DemoDataset(
    demo_root=demo_root,
    mert_processor=mert_processor,
    mert_model=mert_model,
    id_to_label=id_to_label,
)
demo_loader = DataLoader(demo_dataset, batch_size=1, shuffle=False)

# Test the model on demo data
model_demo_test(classifier, demo_loader, device, id_to_label=id_to_label)

Demo File Loaded 4 demo samples
GT: Breathy | Pred: Breathy
GT: Breathy | Pred: Breathy
GT: Vibrato | Pred: Vibrato
GT: Vibrato | Pred: Vibrato

Classifier	Temporal modeling	Complexity	Outcome in our project
1D-CNN	Conv1D over frames + GAP	Low (few params)	Best validation performance; chosen as final model.
CRNN	Conv1D front-end + BiLSTM	Medium–High	Some overfitting; no clear gain over 1D-CNN.
Transformer	Self-attention over all frames	High (most params)	Slowest and least stable; did not beat 1D-CNN.

🎤 Vocal Technique Recognition 🗣️¶

Final Report (MIE1517 Introduction to Deep Learning) - Team 20¶

1. Brief Description of the Project¶

Introduction to Vocal Technique Recognition¶

The Dataset: GTSinger¶

Project Overview and Pipeline¶

2. Approach¶

How to Solve the Problem¶

The Strategy We Are Going to Use¶

Why This Approach Works¶

Loss Function We Optimize¶

3. Implementation of The Vocal Technique Recognizer¶

3-1. Downloading the GTsinger Dataset¶

3-2. Data Splitting Strategy¶

3-3. Visualization of dataset¶

3-4. Model Architecture¶

Data Pre-Processing¶

Feature Extraction with MERT¶

1D-CNN Classification Head¶

Comparison of Classifier Architectures¶

3-5. Loading and Pre-processing Audio Dataset¶

3-6. Vocal Technique Recognizers (Classifier Models)¶

3-7. Train Helpers¶

3-8. Training Configuration¶

3-9 Training Results¶

4. Implementation on New Data¶

4-1. Loading a Trained Model¶

4-2. Model Test Function¶

4-3. Test The Loaded Model with Test Dataset¶

4-4. Test Results on The Test Dataset¶

4-5. Example: Predicting on New Demo Audio Files¶

5. Summary of related work with citations/references¶

5-1 Pre-trained Audio Representation Models¶

5-2 Vocal Technique and Singing Voice Datasets¶

5-3 Transfer Learning for Audio Classification¶

5-4 Transformer Architectures for Audio¶

5-5 Vocal Technique Recognition and Audio Classification¶

5-6 Self-Supervised Learning for Audio¶

6. Discussion¶

Overall Performance Assessment¶

Interesting and Surprising Findings¶

Key Insights and Lessons Learned¶

Limitations and Future Work¶

Conclusion¶